<?php
require_once __DIR__ . '/../../autoloader.php';
use phpspider\core\phpspider;
use phpspider\core\requests;
use phpspider\core\selector;
use phpspider\core\db;


/* Do NOT delete this comment */
/* 不要删除这段注释 */

$configs = array(
'name' => '涿州房屋信息',
'log_show' =>false,
'tasknum' => 4,
//数据库配置
'db_config' => array(
'host'  => '127.0.0.1',
'port'  => 3306,
'user'  => 'root',
'pass'  => 'jbx666',
'name'  => 'pocket_house',
'input_encoding' =>'UTF-8',
'output_encoding'=>'UTF-8'
),
//'export' => array(
//'type' => 'db',
//'table' => 'jiwu',  // 如果数据表没有数据新增请检查表结构和字段名是否匹配
//),
//爬取的域名列表  
'domains' => array(
    'zhuozhou.jiwu.com',
	"imgs2.jiwu.com"
), 
//抓取的起点
'scan_urls' => array(
	'http://zhuozhou.jiwu.com/'
),
'content_url_regexes' => array(
    "http://zhuozhou.jiwu.com/loupan/\d+",

),
'max_try' => 5,

'fields' => array(
    array(
        'name' => "house_name",
        'selector' => "// h1[@class='public-lpm2']",
        'required' => true,
    )
),
);

$spider = new phpspider($configs);

$spider->on_start = function($phpspider) 
{
    $db_config = $phpspider->get_config("db_config");
    // 数据库连接
    db::set_connect('default', $db_config);
    db::_init();
    
    $rows = db::get_all("select * from ph_house where id > 30669");
    for($i=0;$i<count($rows);$i++){
    	$phpspider->add_url("http://zhuozhou.jiwu.com/loupan/".$rows[$i]['phid'].".html");
    }
};


$spider->on_content_page = function($page, $content, $phpspider) 
{
	$content = selector::remove($content, "//div[contains(@class,'footer-v5')]");
    $content = selector::remove($content, "//div[contains(@class,'content-center lp-option-box')]");
    return false;
};

$spider->on_extract_field = function($fieldname, $data, $page) 
{
    $data = selector::remove($data, "//div[contains(@class,'footer-v5')]");
    $data = selector::remove($data, "//div[contains(@class,'content-center lp-option-box')]");
    
    return $data;
};

$spider->on_extract_page = function($page, $data) 
{
	$urls = $page['url'];
	if(strpos($urls,'list-qa')>0){
		echo 'i m 268745';
	}
	else{
		$urlsArr = explode("loupan/",$urls);
		$phid = explode(".",$urlsArr[1])[0];
//		echo '$phid='.$phid."\n";
		
		$html = requests::get($urls);
		
		$house_db = db::get_one("select * from ph_house where phid = $phid");
		if(empty($house_db))
			return false;
		
		//是否是有效数据
		
		
		
		$datarmv = selector::select($html, "//p[contains(@class,'fl lpm-s8-2')]");
		
		/******************标签********************/
		$markhtml = selector::select($html, "//div[contains(@class,'public-lpm4')]");
		$markarr = selector::select($markhtml, "//span");

		$marknum = count($markarr);
		
		$houseid = $house_db['id'];
		$houseoneprice = $house_db['house_unit_price'];
		
		$hmark = db::get_one("select * from ph_house_mark where house_id = $houseid");
		if(empty($hmark)){
			for($i=0;$i<$marknum;$i++){
//				echo 'maskspan='.$markarr[$i];
				$markname = $markarr[$i];
				$markname = str_replace("&#13;\n","",$markname);
				$markname = preg_replace("/[\s]+/is","",$markname);//trim($attvalue,"');
					
				$rows = db::get_one("select * from ph_base_mark where mark like '".$markname."'");
				$markid = 0;
				if(!empty($rows)){
					$markid = $rows['id'];
				}
				else{
					$markid = db::insert('ph_base_mark',['mark'=>$markname,'type'=>1]);
				}
				
				
				db::insert('ph_house_mark',['house_id'=>$houseid,'mark_id'=>$markid]);
			}
		}
		    
	    /***********************获取房屋介绍图片*****************************/
//	    $imgarrs =  selector::select($arrs[$i],"//div[contains(@class,'lpm-section6-slider2')]");
		
		
		$houseimgrow = db::get_one("select * from ph_house_img where hid = $houseid"); 
		if(empty($houseimgrow)){
			
			$imghtml = selector::select($html, "//div[contains(@id,'carsousel2')]");
			$imgarrs = selector::select($imghtml, "//div[contains(@class,'item')]");
			$imgarrs = toArray($imgarrs);
			$imgularrs = selector::select($imgarrs[0], "//ul[contains(@class,'lp-album-s1')]");		
			$topimgurl = selector::select($html, "//img[contains(@class,'lpm-img trs')]");
			$topimgurl = toArray($topimgurl);
			
			$saveimgtopurl = 'upload/house/'.$houseid.'/';
			$topsaveurl = download($topimgurl[0],$saveimgtopurl);
			$topsaveurl = '/'.$saveimgtopurl.$topsaveurl;		
			db::insert('ph_house_img',['hid'=>$houseid,'img_path'=>$topsaveurl]);
			if(count($imgarrs)>0){
				$imgLen = count($imgarrs);
				$addLen = 0;
				for($k = 0;$k<$imgLen;$k++){
					
					$imgliarrs = selector::select($imgarrs[$k], "//li");
					$imgliarrs = toArray($imgliarrs);
					$linum = count($imgliarrs);
					for($i=0;$i<$linum;$i++){
						if($addLen>=5)
							break;
						
						$urldata = selector::select($imgliarrs[$i], "//img");
						if($urldata == '' || $urldata == null)
							continue;
						$urldata = str_replace("_a","_m",$urldata);
						$saveurl = download($urldata,$saveimgtopurl);
						$saveurl = '/'.$saveimgtopurl.$saveurl;
						db::insert('ph_house_img',['hid'=>$houseid,'img_path'=>$saveurl]);
						$addLen+=1;
					}
				
				}			
			}		
		}
		
		/*************************拉取房屋户型图片***************************/
		$housetyperow = db::get_one("select * from ph_house_type_image where house_id = $houseid"); 
		$imgtypehtml = selector::select($html, "//div[contains(@id,'carousel1')]");
		$imgtypearrs = selector::select($imgtypehtml, "//div[contains(@class,'item')]");
		$imgtypearrs = toArray($imgtypearrs);

		if(count($imgtypearrs)>0 && empty($housetyperow)){
			$tpnum = count($imgtypearrs);
			$savetpimgtopurl = 'upload/house_image_type/'.$houseid.'/';
			$housestr = '';
			$smallsize = 9999999;
			$maxsize = 0;
			for($i=0;$i<$tpnum;$i++){
				
				$imgtpArr = selector::select($imgtypearrs[$i], "//div[contains(@class,'house-type-s1')]");
				$imgtpArr = toArray($imgtpArr);
				$tpnumx = count($imgtpArr);
				
				
				
				for($j=0;$j<$tpnumx;$j++){
					$urldata = selector::select($imgtpArr[$j], "//img");
					if($urldata == '' || $urldata == null)
						continue;
						
					$urldata = str_replace("_a","_m",$urldata);
					$saveurl = download($urldata,$savetpimgtopurl);
					$saveurl = '/'.$savetpimgtopurl.$saveurl;
					$housetype = selector::select($imgtpArr[$j], ".house-type-s2 > a", "css");
					$housesize = selector::select($imgtpArr[$j], "//p[contains(@class,'house-type-s3')]");
					
					//首付
					$housepay = selector::select($imgtpArr[$j], "//p[contains(@class,'house-type-s4')]");
					
					$housetype = str_replace("&#13;\n","",$housetype);
					$housetype = preg_replace("/[\s]+/is","",$housetype);
					
					$housesize = str_replace("&#13;\n","",$housesize);
					$housesize = preg_replace("/[\s]+/is","",$housesize);
					
					$housepay = str_replace("&#13;\n","",$housepay);
					$housepay = preg_replace("/[\s]+/is","",$housepay);
					$housepay = str_replace("万","",$housepay);
					
					
					
					$hsize = findNum($housesize);
//					echo '$hsize='.$hsize."\n";
					if($hsize>0 && $smallsize>$hsize){
						$smallsize = $hsize;
					}
					
					if($hsize>0 && $maxsize < $hsize){
						$maxsize = $hsize;
					}
					
					$housestr .= $housetype.'--'.$housesize."--".$housepay."\n";	
					
					$addtp = ['img_path'=>$saveurl,'name'=>$housetype,'house_type'=>$housetype,
					'area'=>$housesize,'price'=>$housepay,'house_id'=>$houseid];
					$imgtpid = db::insert('ph_house_type_image',$addtp);
					
					db::insert('ph_house_type_imglist',['typeid'=>$imgtpid,'img_path'=>$saveurl]);
					
					if($houseoneprice<=0){
						if($housepay > 0 && $hsize>0){
							$houseoneprice = floor((($housepay/0.3)*10000)/$hsize);
						}
					}
				}
				
				
			}
			
			if($smallsize != 9999999  && $maxsize>0){
				$house_area = "'".$smallsize.'-'.$maxsize."'";
				if($smallsize == $maxsize){
					$house_area = $smallsize;
				}
				
				if($smallsize <= 0){
					$house_area = $maxsize;
				}
				
				
				$total_price = ($smallsize * $houseoneprice)/10000;
				db::query("update ph_house set house_unit_price=$houseoneprice,house_area=$house_area, total_price = $total_price where id=$houseid");
			}
			
			$data['housestr'] = $housestr;			
	        
		}
		
//		db::insert("jiwu", $data);
		
		//
//	    echo "image\n";
//	    var_dump($imgtypearrs);
	}
};

$spider->start();

function download($url, $path = 'images/')
{
  $ch = curl_init();
  curl_setopt($ch, CURLOPT_URL, $url);
  curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
  curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 30);
  $file = curl_exec($ch);
  curl_close($ch);
  
  if(!file_exists($path)){
  	mkdir($path);
  }
  
  $filename = pathinfo($url, PATHINFO_BASENAME);
  $resource = fopen($path . $filename, 'a');
  fwrite($resource, $file);
  fclose($resource);
  return $filename;
}

function findNum($str=''){
    $str=trim($str);
    if(empty($str)){return '';}
    
    $reg = "/(\d+)/";
    preg_match_all($reg,$str,$result);
    if(is_array($result)&&!empty($result)&&!empty($result[1])&&!empty($result[1][0])){
        return $result[1][0];
    }
    return '';
}

function toArray($parr){	
	if(is_array($parr)){
		return $parr;
	}
	else{
		return [$parr];
	}
}